{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 128
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 20066,
     "status": "ok",
     "timestamp": 1597244667844,
     "user": {
      "displayName": "frank kk",
      "photoUrl": "",
      "userId": "02779524038085974778"
     },
     "user_tz": -540
    },
    "id": "4wBUoSK_89ms",
    "outputId": "ebb34211-238d-4974-dd07-853c7aebc5e4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly&response_type=code\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 655
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9917,
     "status": "ok",
     "timestamp": 1597244769451,
     "user": {
      "displayName": "frank kk",
      "photoUrl": "",
      "userId": "02779524038085974778"
     },
     "user_tz": -540
    },
    "id": "7hKbigSd97R1",
    "outputId": "b424ccf6-ca33-4bb1-afc8-330227ae1914"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/27/3c/91ed8f5c4e7ef3227b4119200fc0ed4b4fd965b1f0172021c25701087825/transformers-3.0.2-py3-none-any.whl (769kB)\n",
      "\u001b[K     |████████████████████████████████| 778kB 2.8MB/s \n",
      "\u001b[?25hCollecting sentencepiece!=0.1.92\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
      "\u001b[K     |████████████████████████████████| 1.1MB 15.6MB/s \n",
      "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
      "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
      "Collecting tokenizers==0.8.1.rc1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/40/d0/30d5f8d221a0ed981a186c8eb986ce1c94e3a6e87f994eae9f4aa5250217/tokenizers-0.8.1rc1-cp36-cp36m-manylinux1_x86_64.whl (3.0MB)\n",
      "\u001b[K     |████████████████████████████████| 3.0MB 19.3MB/s \n",
      "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
      "Collecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
      "\u001b[K     |████████████████████████████████| 890kB 37.0MB/s \n",
      "\u001b[?25hRequirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (1.15.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=9ff67ad561aede7292c4f1d7f82736acbc397211c1caed226fcb0ba01b3f9bd3\n",
      "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: sentencepiece, tokenizers, sacremoses, transformers\n",
      "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.8.1rc1 transformers-3.0.2\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 847,
     "status": "ok",
     "timestamp": 1597244771400,
     "user": {
      "displayName": "frank kk",
      "photoUrl": "",
      "userId": "02779524038085974778"
     },
     "user_tz": -540
    },
    "id": "uavgJ5Du-GyE",
    "outputId": "dba29e53-bf2d-4146-fa44-335e3a96034f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Errno 2] No such file or directory: 'gdrive/My Drive/'\n",
      "/content/gdrive/My Drive\n"
     ]
    }
   ],
   "source": [
    "% cd gdrive/My Drive/\n",
    "import sys\n",
    "sys.path.append('content/gdrive/My Drive/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 7129,
     "status": "ok",
     "timestamp": 1597244781025,
     "user": {
      "displayName": "frank kk",
      "photoUrl": "",
      "userId": "02779524038085974778"
     },
     "user_tz": -540
    },
    "id": "TXSBNw1j9iFN"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import datetime\n",
    "import pkg_resources\n",
    "import random\n",
    "import time\n",
    "import scipy.stats as stats\n",
    "import gc\n",
    "import re\n",
    "import operator\n",
    "import sys\n",
    "from sklearn import metrics\n",
    "from sklearn import model_selection\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.utils.data\n",
    "import torch.nn.functional as F\n",
    "from nltk.stem import PorterStemmer\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from tqdm import tqdm, tqdm_notebook\n",
    "import os\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "import time\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "import warnings\n",
    "import pickle\n",
    "#from apex import amp\n",
    "import shutil\n",
    "\n",
    "from torch.utils.data import TensorDataset\n",
    "from transformers import BertConfig, BertForSequenceClassification, BertTokenizer\n",
    "from transformers import AdamW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1372,
     "status": "ok",
     "timestamp": 1597244796794,
     "user": {
      "displayName": "frank kk",
      "photoUrl": "",
      "userId": "02779524038085974778"
     },
     "user_tz": -540
    },
    "id": "LwDL9a0u-QAa",
    "outputId": "3e65abaa-1d56-4e0b-8143-9a2095186a0a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7f7308ae4540>"
      ]
     },
     "execution_count": 8,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "warnings.filterwarnings(action='once')\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "MAX_SEQUENCE_LENGTH = 220\n",
    "SEED = 1234\n",
    "BATCH_SIZE = 32\n",
    "BERT_MODEL_PATH = 'bert-base-uncased'\n",
    "CONFIG_PATH = 'bert-base-uncased'\n",
    "\n",
    "Data_dir = \"./jigsaw-unintended-bias-in-toxicity-classification\"\n",
    "#Input_dir = \"../input\"\n",
    "#Work_dir = \"../working/\"\n",
    "\n",
    "num_to_load = 1000000                         #Train size to match time limit\n",
    "valid_size = 100000                          #Validation Size\n",
    "\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.cuda.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "TARGET_COLUMN = 'target'\n",
    "TEXT_COLUMN = 'comment_text'\n",
    "\n",
    "identity_columns = [\n",
    "    'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',\n",
    "    'muslim', 'black', 'white', 'psychiatric_or_mental_illness']\n",
    "\n",
    "AUX_COLUMNS = ['target', 'severe_toxicity','obscene','identity_attack','insult','threat','sexual_explicit']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1028,
     "status": "ok",
     "timestamp": 1597244799186,
     "user": {
      "displayName": "frank kk",
      "photoUrl": "",
      "userId": "02779524038085974778"
     },
     "user_tz": -540
    },
    "id": "WOs90o4J-fRc"
   },
   "outputs": [],
   "source": [
    "# 处理数据 comment_text\n",
    "def convert_to_feature(tokenizer, train_data):\n",
    "    q_tokens = []\n",
    "    max_seq_len = 0\n",
    "    for text in tqdm(train_data):\n",
    "        bert_ids = tokenizer.build_inputs_with_special_tokens(tokenizer.convert_tokens_to_ids(tokenizer.tokenize(text)))\n",
    "        if len(bert_ids) > max_seq_len:\n",
    "            max_seq_len = len(bert_ids)\n",
    "        q_tokens.append(bert_ids)\n",
    "\n",
    "\n",
    "    print(\"max len of questions :\", max_seq_len)\n",
    "    assert max_seq_len <= 512  # BERT-base限制长度512\n",
    "\n",
    "    # padding length with '0'\n",
    "    for q in q_tokens:\n",
    "        while len(q) < max_seq_len:\n",
    "            q.append(0)\n",
    "\n",
    "    '''\n",
    "    # BERT input embedding\n",
    "    # 每个位置 [1,1,1,1, 1,1,1,1, 1,1,1,1]\n",
    "    input_masks = [[1] * max_seq_len for _ in range(len(train_data))]\n",
    "\n",
    "    # 每个句子 [0,0,0,0, 1,1,1,1, 2,2,2,2]\n",
    "    input_segment_ids = [[0] * max_seq_len for _ in range(len(train_data))]\n",
    "    assert len(q_tokens) == len(train_data) and \\\n",
    "           len(q_tokens) == len(input_masks) and \\\n",
    "           len(q_tokens) == len(input_segment_ids)\n",
    "    \n",
    "\n",
    "    data_features = {'input_ids': q_tokens,\n",
    "                     'input_masks': input_masks,\n",
    "                     'input_segment_ids': input_segment_ids\n",
    "                     #'y_labels': y_labels\n",
    "                     }\n",
    "    \n",
    "    '''\n",
    "    return q_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 800,
     "status": "ok",
     "timestamp": 1597244801370,
     "user": {
      "displayName": "frank kk",
      "photoUrl": "",
      "userId": "02779524038085974778"
     },
     "user_tz": -540
    },
    "id": "9FmMmlSP-qag"
   },
   "outputs": [],
   "source": [
    "def preprocess(data):\n",
    "    '''\n",
    "    Credit goes to https://www.kaggle.com/gpreda/jigsaw-fast-compact-solution\n",
    "    '''\n",
    "    punct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~`\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'\n",
    "    def clean_special_chars(text, punct):\n",
    "        for p in punct:\n",
    "            text = text.replace(p, ' ')\n",
    "        return text\n",
    "\n",
    "    data = data.astype(str).apply(lambda x: clean_special_chars(x, punct))\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 19107,
     "status": "ok",
     "timestamp": 1597244823167,
     "user": {
      "displayName": "frank kk",
      "photoUrl": "",
      "userId": "02779524038085974778"
     },
     "user_tz": -540
    },
    "id": "K1bLOPKY-uXG",
    "outputId": "9182e942-994b-4808-e697-ce71cf90d616"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loaded 1100000 records\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv('/content/gdrive/My Drive/public/data/train.csv')\n",
    "#test_df = pd.read_csv('/content/gdrive/My Drive/public/data/test.csv')\n",
    "train_df = train_df.sample(num_to_load+valid_size, random_state=SEED)\n",
    "print('loaded %d records' % len(train_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1288,
     "status": "ok",
     "timestamp": 1597242338620,
     "user": {
      "displayName": "frank kk",
      "photoUrl": "",
      "userId": "02779524038085974778"
     },
     "user_tz": -540
    },
    "id": "rKy58Ux2p4yh"
   },
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, do_lower_case=True, return_tensors='pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 106664,
     "status": "ok",
     "timestamp": 1597231542360,
     "user": {
      "displayName": "Cecilia SS",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhZdujNRISiRFR_Nh34nbB8-fEtGSvGXr2rEpf5=s64",
      "userId": "04154856476552279883"
     },
     "user_tz": -540
    },
    "id": "mMsR4CZjUmM6",
    "outputId": "a964155b-bff0-445d-b81f-48bff6cd5459"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 97320/97320 [01:41<00:00, 961.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max len of questions : 357\n"
     ]
    }
   ],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, do_lower_case=True, return_tensors='pt')\n",
    "x_test = preprocess(test_df[TEXT_COLUMN].fillna(\"DUMMY_VALUE\"))\n",
    "test_tokens = convert_to_feature(tokenizer, x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aZoLgkR3VWRW"
   },
   "outputs": [],
   "source": [
    "if not os.path.isdir(\"/content/gdrive/My Drive/nlpclass/embeddings/\"):\n",
    "    os.mkdir(\"/content/gdrive/My Drive/nlpclass/embeddings/\")\n",
    "#output1 = open('/content/gdrive/My Drive/nlpclass/embeddings/train_tokens.pkl', 'wb')\n",
    "output2 = open('/content/gdrive/My Drive/nlpclass/embeddings/test_tokens.pkl', 'wb')\n",
    "#pickle.dump(train_tokens, output1)  # pickle之后读取比txt效率高\n",
    "pickle.dump(test_tokens, output2)  # pickle之后读取比txt效率高"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LMsLleZU_BFy"
   },
   "outputs": [],
   "source": [
    "x_train = preprocess(train_df[TEXT_COLUMN].fillna(\"DUMMY_VALUE\"))\n",
    "#x_test = preprocess(test_df[TEXT_COLUMN].fillna(\"DUMMY_VALUE\"))\n",
    "\n",
    "#tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, do_lower_case=True, return_tensors='pt')\n",
    "\n",
    "train_tokens = convert_to_feature(tokenizer, x_train)\n",
    "\n",
    "train_df = train_df.fillna(0)\n",
    "y_train = np.where(train_df[TARGET_COLUMN] >= 0.5, 1, 0)\n",
    "y_aux_train = train_df[AUX_COLUMNS].values\n",
    "\n",
    "#test_tokens = convert_to_feature(tokenizer, x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "msiIKoHiTjpl"
   },
   "outputs": [],
   "source": [
    "if not os.path.isdir(\"/content/gdrive/My Drive/nlpclass/embeddings/\"):\n",
    "    os.mkdir(\"/content/gdrive/My Drive/nlpclass/embeddings/\")\n",
    "output1 = open('/content/gdrive/My Drive/nlpclass/embeddings/train_tokens.pkl', 'wb')\n",
    "#output2 = open('/content/gdrive/My Drive/nlpclass/embeddings/test_tokens.pkl', 'wb')\n",
    "pickle.dump(train_tokens, output1)  # pickle之后读取比txt效率高\n",
    "#pickle.dump(test_tokens, output2)  # pickle之后读取比txt效率高"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1085120,
     "status": "ok",
     "timestamp": 1597232640640,
     "user": {
      "displayName": "Cecilia SS",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhZdujNRISiRFR_Nh34nbB8-fEtGSvGXr2rEpf5=s64",
      "userId": "04154856476552279883"
     },
     "user_tz": -540
    },
    "id": "bxcqTwZBXb5K",
    "outputId": "5290c31a-bb36-4373-8765-df0e943f7522"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000000/1000000 [17:19<00:00, 962.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max len of questions : 335\n"
     ]
    }
   ],
   "source": [
    "x_train = preprocess(train_df[TEXT_COLUMN].fillna(\"DUMMY_VALUE\"))\n",
    "\n",
    "train_tokens = convert_to_feature(tokenizer, x_train[:num_to_load])\n",
    "if not os.path.isdir(\"/content/gdrive/My Drive/nlpclass/embeddings/\"):\n",
    "    os.mkdir(\"/content/gdrive/My Drive/nlpclass/embeddings/\")\n",
    "output1 = open('/content/gdrive/My Drive/nlpclass/embeddings/train_tokens.pkl', 'wb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 109872,
     "status": "ok",
     "timestamp": 1597232925423,
     "user": {
      "displayName": "Cecilia SS",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhZdujNRISiRFR_Nh34nbB8-fEtGSvGXr2rEpf5=s64",
      "userId": "04154856476552279883"
     },
     "user_tz": -540
    },
    "id": "39A7GCqmZGGV",
    "outputId": "3713afd4-b5ca-4777-8010-7e02b4ca1968"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100000/100000 [01:45<00:00, 949.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max len of questions : 320\n"
     ]
    }
   ],
   "source": [
    "valid_tokens = convert_to_feature(tokenizer, x_train[num_to_load:])\n",
    "if not os.path.isdir(\"/content/gdrive/My Drive/nlpclass/embeddings/\"):\n",
    "    os.mkdir(\"/content/gdrive/My Drive/nlpclass/embeddings/\")\n",
    "output3 = open('/content/gdrive/My Drive/nlpclass/embeddings/valid_tokens.pkl', 'wb')\n",
    "\n",
    "train_df = train_df.fillna(0)\n",
    "y_train = np.where(train_df[TARGET_COLUMN] >= 0.5, 1, 0)\n",
    "#y_aux_train = train_df[AUX_COLUMNS].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_Kyzod3cf0Vp"
   },
   "outputs": [],
   "source": [
    "pickle.dump(train_tokens, output1)  # pickle之后读取比txt效率高\n",
    "pickle.dump(valid_tokens, output3)  # pickle之后读取比txt效率高"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 21781,
     "status": "ok",
     "timestamp": 1597242380824,
     "user": {
      "displayName": "frank kk",
      "photoUrl": "",
      "userId": "02779524038085974778"
     },
     "user_tz": -540
    },
    "id": "7J9UiUgEZa_U"
   },
   "outputs": [],
   "source": [
    "# restart --ver\n",
    "#pkl_file1 = open('/content/gdrive/My Drive/embeding/test_tokens.pkl', 'rb')\n",
    "#test_tokens = pickle.load(pkl_file1)\n",
    "pkl_file2 = open('/content/gdrive/My Drive/embeding/train_tokens.pkl', 'rb')\n",
    "train_tokens = pickle.load(pkl_file2)\n",
    "#pkl_file3 = open('/content/gdrive/My Drive/embeding/valid_tokens.pkl', 'rb')\n",
    "#valid_tokens = pickle.load(pkl_file3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8xjO1sHa_G7E"
   },
   "outputs": [],
   "source": [
    "# split train and validation dataset\n",
    "#X = train_X[:num_to_load]     \n",
    "#X = train_tokens            \n",
    "#y = y_train.values[:num_to_load]\n",
    "#X_val = train_X[num_to_load:]   \n",
    "#X_val = train_X[num_to_load:]             \n",
    "#y_val = y_train.values[num_to_load:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2057,
     "status": "ok",
     "timestamp": 1597242445208,
     "user": {
      "displayName": "frank kk",
      "photoUrl": "",
      "userId": "02779524038085974778"
     },
     "user_tz": -540
    },
    "id": "ATO3MAKbYWxn"
   },
   "outputs": [],
   "source": [
    "# split train and validation dataset -- ver2\n",
    "train_df = train_df.fillna(0)\n",
    "y_train = np.where(train_df[TARGET_COLUMN] >= 0.5, 1, 0)\n",
    "\n",
    "X = train_tokens     \n",
    "y = y_train[:num_to_load]\n",
    "#X_val = valid_tokens       \n",
    "#y_val = y_train[num_to_load:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 262
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1151,
     "status": "error",
     "timestamp": 1597242490586,
     "user": {
      "displayName": "frank kk",
      "photoUrl": "",
      "userId": "02779524038085974778"
     },
     "user_tz": -540
    },
    "id": "UpV2n9A__MhO",
    "outputId": "d42649f1-8f8c-407b-8022-72ba984168cf"
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-055e97c3fb45>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mweights\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mones\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_df\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;31m# Subgroup  identity_columns  > 0.5\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mweights\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtrain_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midentity_columns\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfillna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m>=\u001b[0m\u001b[0;36m0.5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# Background Positive, Subgroup Negative\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m weights += (( (train_df[TARGET_COLUMN].values>=0.5).astype(bool).astype(np.int) +\n",
      "\u001b[0;31mNameError\u001b[0m: name 'np' is not defined"
     ]
    }
   ],
   "source": [
    "weights = np.ones((len(train_df),)) / 4\n",
    "# Subgroup  identity_columns  > 0.5\n",
    "weights += (train_df[identity_columns].fillna(0).values>=0.5).sum(axis=1).astype(bool).astype(np.int) / 4\n",
    "# Background Positive, Subgroup Negative\n",
    "weights += (( (train_df[TARGET_COLUMN].values>=0.5).astype(bool).astype(np.int) +\n",
    "   (train_df[identity_columns].fillna(0).values<0.5).sum(axis=1).astype(bool).astype(np.int) ) > 1 ).astype(bool).astype(np.int) / 4\n",
    "# Background Negative, Subgroup Positive\n",
    "weights += (( (train_df[TARGET_COLUMN].values<0.5).astype(bool).astype(np.int) +\n",
    "   (train_df[identity_columns].fillna(0).values>=0.5).sum(axis=1).astype(bool).astype(np.int) ) > 1 ).astype(bool).astype(np.int) / 4\n",
    "\n",
    "#coll = ['black','white','homosexual_gay_or_lesbian','muslim']\n",
    "#weights += (( (train_df[TARGET_COLUMN].values>=0.5).astype(bool).astype(np.int) +\n",
    "#   (train_df[coll].fillna(0).values<0.5).sum(axis=1).astype(bool).astype(np.int) ) > 1 ).astype(bool).astype(np.int) /8\n",
    "\n",
    "#weights += (( (train_df[TARGET_COLUMN].values<0.5).astype(bool).astype(np.int) +\n",
    "#   (train_df[coll].fillna(0).values>=0.5).sum(axis=1).astype(bool).astype(np.int) ) > 1 ).astype(bool).astype(np.int) / 8\n",
    "\n",
    "loss_weight = 1.0 / weights.mean()\n",
    "weights = weights.reshape(-1,1)\n",
    "y.reshape\n",
    "#train_df = train_df.drop(['comment_text'], axis=1)\n",
    "\n",
    "print(weights[:num_to_load].shape, y.shape)\n",
    "y = np.hstack([weights[:num_to_load],y.reshape(-1,1)]) #列项拼接 多标签y\n",
    "#y_val = np.hstack([weights[num_to_load:],y_val.reshape(-1,1)]) #列项拼接 多标签y\n",
    "import gc\n",
    "del weights\n",
    "del train_df \n",
    "del AUX_COLUMNS\n",
    "gc.collect()\n",
    "\n",
    "train_dataset = torch.utils.data.TensorDataset(torch.tensor(X, dtype=torch.long), torch.tensor(y, dtype=torch.float))\n",
    "#valid_dataset = torch.utils.data.TensorDataset(torch.tensor(X_val, dtype=torch.long), torch.tensor(y_val, dtype=torch.float))\n",
    "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=16, shuffle=True, num_workers=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 944,
     "status": "ok",
     "timestamp": 1597241508426,
     "user": {
      "displayName": "frank kk",
      "photoUrl": "",
      "userId": "02779524038085974778"
     },
     "user_tz": -540
    },
    "id": "MSAgiBoC_RbS"
   },
   "outputs": [],
   "source": [
    "# cross entropy\n",
    "def custom_loss(data, targets):\n",
    "    ''' Define custom loss function for weighted BCE on 'target' column '''\n",
    "    out = nn.BCEWithLogitsLoss(weight=targets[:,0])(data[:,0], targets[:,1])\n",
    "    #bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,0:1])(data[:,0:1],targets[:,1:2])\n",
    "    #bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:]) \n",
    "    return out * loss_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4378,
     "status": "ok",
     "timestamp": 1597241523155,
     "user": {
      "displayName": "frank kk",
      "photoUrl": "",
      "userId": "02779524038085974778"
     },
     "user_tz": -540
    },
    "id": "FF6sWE3C9bnW",
    "outputId": "28957e73-766f-441d-e8ef-aa408471d279"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "144"
      ]
     },
     "execution_count": 14,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "del train_dataset \n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "id": "JPwfA9sf_VuF",
    "outputId": "813d8127-cf7e-4582-e15d-62e9199b8fd0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7f68427c24e0>"
      ]
     },
     "execution_count": 14,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_model_file = \"/content/gdrive/My Drive/nlpclass/trained_model/bert_pytorch.bin\"\n",
    "\n",
    "lr=2e-5\n",
    "batch_size = 6\n",
    "#accumulation_steps=2\n",
    "EPOCHS=2\n",
    "\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.cuda.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "config = BertConfig.from_pretrained(BERT_MODEL_PATH, num_labels=1)\n",
    "model = BertForSequenceClassification.from_pretrained(BERT_MODEL_PATH, from_tf=True, config=config)\n",
    "model.zero_grad()\n",
    "model = model.to(device)\n",
    "param_optimizer = list(model.named_parameters())\n",
    "no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
    "optimizer_grouped_parameters = [\n",
    "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
    "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
    "    ]\n",
    "#train = train_dataset\n",
    "'''\n",
    "num_train_optimization_steps = int(EPOCHS*len(train)/batch_size/accumulation_steps)\n",
    "\n",
    "optimizer = AdamW(optimizer_grouped_parameters,\n",
    "                  lr=lr,\n",
    "                  warmup=0.05,\n",
    "                  t_total=num_train_optimization_steps)\n",
    "model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\", verbosity=0)\n",
    "'''\n",
    "optimizer = AdamW(optimizer_grouped_parameters, lr=lr, eps=1e-7)\n",
    "optimizer.zero_grad()\n",
    "model = model.train()\n",
    "\n",
    "tq = tqdm_notebook(range(EPOCHS))\n",
    "for epoch in tq:\n",
    "    \n",
    "    total_steps = len(train_loader)\n",
    "    avg_loss = 0.\n",
    "    avg_acc = 0.\n",
    "    lossf=None\n",
    "    tk0 = tqdm_notebook(enumerate(train_loader), total=len(train_loader),leave=False)\n",
    "    optimizer.zero_grad()   \n",
    "\n",
    "    if epoch == 1 :\n",
    "        for param_group in optimizer.param_groups:\n",
    "            param_group['lr'] = 1e-5\n",
    "\n",
    "    for i, batch in tk0:\n",
    "        x_batch, y_batch = tuple(t.to(device) for t in batch)\n",
    "        output = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
    "        y_pred, logits = output[:2] \n",
    "        loss = custom_loss(y_pred, y_batch.to(device))        \n",
    "        '''\n",
    "        with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "            scaled_loss.backward()\n",
    "        \n",
    "        if (i+1) % accumulation_steps == 0:             # Wait for several backward steps\n",
    "            optimizer.step()                            # Now we can do an optimizer step\n",
    "            optimizer.zero_grad()\n",
    "        '''\n",
    "        optimizer.zero_grad()\n",
    "        loss.sum().backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        if lossf:\n",
    "            lossf = 0.98*lossf+0.02*loss.item()\n",
    "        else:\n",
    "            lossf = loss.item()\n",
    "        \n",
    "        tk0.set_postfix(loss=lossf)\n",
    "        avg_loss += loss.item() / len(train_loader)\n",
    "        avg_acc += torch.mean(((torch.sigmoid(y_pred[:,0])>0.5) == (y_batch[:,1]>0.5).to(device)).to(torch.float) ).item()/len(train_loader)\n",
    "        \n",
    "        # log\n",
    "        if (i + 1) % 100 == 0:\n",
    "            print('Epoch [{}/{}], Step [{}/{}], TrainLoss: {:.4f}, TrainAcc: {:.4f}'\\\n",
    "                  .format(epoch+1, EPOCHS, i+1, total_steps, avg_loss, avg_acc))\n",
    "        \n",
    "            # checkpoint: record model parameter of each epoch\n",
    "            checkpoint = {\"net\": model.state_dict(),\n",
    "                          \"optimizer\": optimizer.state_dict(),\n",
    "                          \"epoch\": epoch}\n",
    "\n",
    "            if not os.path.isdir(\"/content/gdrive/My Drive/nlpclass/model_parameter/\"):\n",
    "                os.mkdir(\"/content/gdrive/My Drive/nlpclass/model_parameter/\")\n",
    "            torch.save(checkpoint, '/content/gdrive/My Drive/nlpclass/model_parameter/ckpt_%s.pth' % (str(epoch)))\n",
    "        '''\n",
    "        tk0.set_postfix(loss=lossf)\n",
    "        avg_loss += loss.item() / len(train_loader)\n",
    "        avg_acc += torch.mean(((torch.sigmoid(y_pred[:,0])>0.5) == (y_batch[:,0]>0.5).to(device)).to(torch.float) ).item()/len(train_loader)\n",
    "        '''\n",
    "    tq.set_postfix(avg_loss=avg_loss,avg_accuracy=avg_acc)\n",
    "\n",
    "torch.save(model.state_dict(), output_model_file)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "bert-toixc.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
